{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 1: Setting Up Development Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: openai in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (1.23.1)\n",
      "Requirement already satisfied: tiktoken in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (0.6.0)\n",
      "Requirement already satisfied: llama_hub in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (0.0.45)\n",
      "Requirement already satisfied: llama_index in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (0.10.30)\n",
      "Requirement already satisfied: pypdf in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (4.2.0)\n",
      "Requirement already satisfied: accelerate in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (0.29.3)\n",
      "Requirement already satisfied: sentence_transformers in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (2.7.0)\n",
      "Requirement already satisfied: neo4j in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (5.19.0)\n",
      "Requirement already satisfied: anyio<5,>=3.5.0 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from openai) (3.7.1)\n",
      "Requirement already satisfied: distro<2,>=1.7.0 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from openai) (1.8.0)\n",
      "Requirement already satisfied: httpx<1,>=0.23.0 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from openai) (0.25.2)\n",
      "Requirement already satisfied: pydantic<3,>=1.9.0 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from openai) (2.5.2)\n",
      "Requirement already satisfied: sniffio in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from openai) (1.3.0)\n",
      "Requirement already satisfied: tqdm>4 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from openai) (4.66.1)\n",
      "Requirement already satisfied: typing-extensions<5,>=4.7 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from openai) (4.11.0)\n",
      "Requirement already satisfied: regex>=2022.1.18 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from tiktoken) (2023.10.3)\n",
      "Requirement already satisfied: requests>=2.26.0 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from tiktoken) (2.31.0)\n",
      "Requirement already satisfied: atlassian-python-api in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from llama_hub) (3.41.11)\n",
      "Requirement already satisfied: html2text in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from llama_hub) (2024.2.26)\n",
      "Requirement already satisfied: psutil in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from llama_hub) (5.9.0)\n",
      "Requirement already satisfied: retrying in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from llama_hub) (1.3.4)\n",
      "Requirement already satisfied: llama-index-agent-openai<0.3.0,>=0.1.4 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from llama_index) (0.2.2)\n",
      "Requirement already satisfied: llama-index-cli<0.2.0,>=0.1.2 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from llama_index) (0.1.12)\n",
      "Requirement already satisfied: llama-index-core<0.11.0,>=0.10.30 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from llama_index) (0.10.30)\n",
      "Requirement already satisfied: llama-index-embeddings-openai<0.2.0,>=0.1.5 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from llama_index) (0.1.7)\n",
      "Requirement already satisfied: llama-index-indices-managed-llama-cloud<0.2.0,>=0.1.2 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from llama_index) (0.1.5)\n",
      "Requirement already satisfied: llama-index-legacy<0.10.0,>=0.9.48 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from llama_index) (0.9.48)\n",
      "Requirement already satisfied: llama-index-llms-openai<0.2.0,>=0.1.13 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from llama_index) (0.1.15)\n",
      "Requirement already satisfied: llama-index-multi-modal-llms-openai<0.2.0,>=0.1.3 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from llama_index) (0.1.5)\n",
      "Requirement already satisfied: llama-index-program-openai<0.2.0,>=0.1.3 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from llama_index) (0.1.5)\n",
      "Requirement already satisfied: llama-index-question-gen-openai<0.2.0,>=0.1.2 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from llama_index) (0.1.3)\n",
      "Requirement already satisfied: llama-index-readers-file<0.2.0,>=0.1.4 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from llama_index) (0.1.19)\n",
      "Requirement already satisfied: llama-index-readers-llama-parse<0.2.0,>=0.1.2 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from llama_index) (0.1.4)\n",
      "Requirement already satisfied: numpy>=1.17 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from accelerate) (1.26.2)\n",
      "Requirement already satisfied: packaging>=20.0 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from accelerate) (24.0)\n",
      "Requirement already satisfied: pyyaml in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from accelerate) (6.0.1)\n",
      "Requirement already satisfied: torch>=1.10.0 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from accelerate) (2.2.0.dev20231208)\n",
      "Requirement already satisfied: huggingface-hub in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from accelerate) (0.19.4)\n",
      "Requirement already satisfied: safetensors>=0.3.1 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from accelerate) (0.4.1)\n",
      "Requirement already satisfied: transformers<5.0.0,>=4.34.0 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from sentence_transformers) (4.35.2)\n",
      "Requirement already satisfied: scikit-learn in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from sentence_transformers) (1.3.2)\n",
      "Requirement already satisfied: scipy in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from sentence_transformers) (1.11.4)\n",
      "Requirement already satisfied: Pillow in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from sentence_transformers) (10.1.0)\n",
      "Requirement already satisfied: pytz in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from neo4j) (2023.3.post1)\n",
      "Requirement already satisfied: idna>=2.8 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from anyio<5,>=3.5.0->openai) (3.6)\n",
      "Requirement already satisfied: certifi in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from httpx<1,>=0.23.0->openai) (2023.11.17)\n",
      "Requirement already satisfied: httpcore==1.* in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from httpx<1,>=0.23.0->openai) (1.0.2)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai) (0.14.0)\n",
      "Requirement already satisfied: filelock in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from huggingface-hub->accelerate) (3.13.1)\n",
      "Requirement already satisfied: fsspec>=2023.5.0 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from huggingface-hub->accelerate) (2023.12.1)\n",
      "Requirement already satisfied: SQLAlchemy>=1.4.49 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.11.0,>=0.10.30->llama_index) (2.0.23)\n",
      "Requirement already satisfied: aiohttp<4.0.0,>=3.8.6 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.30->llama_index) (3.9.1)\n",
      "Requirement already satisfied: dataclasses-json in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.30->llama_index) (0.6.3)\n",
      "Requirement already satisfied: deprecated>=1.2.9.3 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.30->llama_index) (1.2.14)\n",
      "Requirement already satisfied: dirtyjson<2.0.0,>=1.0.8 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.30->llama_index) (1.0.8)\n",
      "Requirement already satisfied: llamaindex-py-client<0.2.0,>=0.1.18 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.30->llama_index) (0.1.18)\n",
      "Requirement already satisfied: nest-asyncio<2.0.0,>=1.5.8 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.30->llama_index) (1.6.0)\n",
      "Requirement already satisfied: networkx>=3.0 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.30->llama_index) (3.2.1)\n",
      "Requirement already satisfied: nltk<4.0.0,>=3.8.1 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.30->llama_index) (3.8.1)\n",
      "Requirement already satisfied: pandas in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.30->llama_index) (2.1.3)\n",
      "Requirement already satisfied: tenacity<9.0.0,>=8.2.0 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.30->llama_index) (8.2.3)\n",
      "Requirement already satisfied: typing-inspect>=0.8.0 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.30->llama_index) (0.9.0)\n",
      "Requirement already satisfied: wrapt in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from llama-index-core<0.11.0,>=0.10.30->llama_index) (1.16.0)\n",
      "Requirement already satisfied: beautifulsoup4<5.0.0,>=4.12.3 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from llama-index-readers-file<0.2.0,>=0.1.4->llama_index) (4.12.3)\n",
      "Requirement already satisfied: striprtf<0.0.27,>=0.0.26 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from llama-index-readers-file<0.2.0,>=0.1.4->llama_index) (0.0.26)\n",
      "Requirement already satisfied: llama-parse<0.5.0,>=0.4.0 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from llama-index-readers-llama-parse<0.2.0,>=0.1.2->llama_index) (0.4.1)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from pydantic<3,>=1.9.0->openai) (0.6.0)\n",
      "Requirement already satisfied: pydantic-core==2.14.5 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from pydantic<3,>=1.9.0->openai) (2.14.5)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from requests>=2.26.0->tiktoken) (3.3.2)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from requests>=2.26.0->tiktoken) (2.1.0)\n",
      "Requirement already satisfied: sympy in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from torch>=1.10.0->accelerate) (1.12)\n",
      "Requirement already satisfied: jinja2 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from torch>=1.10.0->accelerate) (3.1.2)\n",
      "Requirement already satisfied: tokenizers<0.19,>=0.14 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from transformers<5.0.0,>=4.34.0->sentence_transformers) (0.15.0)\n",
      "Requirement already satisfied: six in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from atlassian-python-api->llama_hub) (1.16.0)\n",
      "Requirement already satisfied: oauthlib in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from atlassian-python-api->llama_hub) (3.2.2)\n",
      "Requirement already satisfied: requests-oauthlib in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from atlassian-python-api->llama_hub) (2.0.0)\n",
      "Requirement already satisfied: jmespath in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from atlassian-python-api->llama_hub) (1.0.1)\n",
      "Requirement already satisfied: joblib>=1.1.1 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from scikit-learn->sentence_transformers) (1.3.2)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from scikit-learn->sentence_transformers) (3.2.0)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.11.0,>=0.10.30->llama_index) (23.1.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.11.0,>=0.10.30->llama_index) (6.0.4)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.11.0,>=0.10.30->llama_index) (1.9.4)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.11.0,>=0.10.30->llama_index) (1.4.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from aiohttp<4.0.0,>=3.8.6->llama-index-core<0.11.0,>=0.10.30->llama_index) (1.3.1)\n",
      "Requirement already satisfied: soupsieve>1.2 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from beautifulsoup4<5.0.0,>=4.12.3->llama-index-readers-file<0.2.0,>=0.1.4->llama_index) (2.5)\n",
      "Requirement already satisfied: click in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from nltk<4.0.0,>=3.8.1->llama-index-core<0.11.0,>=0.10.30->llama_index) (8.1.7)\n",
      "Requirement already satisfied: greenlet!=0.4.17 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from SQLAlchemy[asyncio]>=1.4.49->llama-index-core<0.11.0,>=0.10.30->llama_index) (3.0.3)\n",
      "Requirement already satisfied: mypy-extensions>=0.3.0 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from typing-inspect>=0.8.0->llama-index-core<0.11.0,>=0.10.30->llama_index) (1.0.0)\n",
      "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from dataclasses-json->llama-index-core<0.11.0,>=0.10.30->llama_index) (3.20.1)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.3)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from pandas->llama-index-core<0.11.0,>=0.10.30->llama_index) (2.9.0)\n",
      "Requirement already satisfied: tzdata>=2022.1 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from pandas->llama-index-core<0.11.0,>=0.10.30->llama_index) (2023.3)\n",
      "Requirement already satisfied: mpmath>=0.19 in /Users/minp/miniconda3/envs/KnowledgeGraph/lib/python3.12/site-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "#markdown Install the libraries:\n",
    "%pip install openai tiktoken llama_hub llama_index pypdf accelerate sentence_transformers llama_index neo4j "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 提前准备\n",
    "- Docker Desktop\n",
    "- Azure OpenAI 或者 GPT-4 （免费的 OpenAPI GPT-3.5 有 token 限制）\n",
    "- python 版本小于 3.12（本项目使用 3.11.6python 版本）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "cannot import name 'AzureOpenAI' from 'llama_index.llms' (unknown location)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[2], line 9\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpathlib\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Path\n\u001b[1;32m      7\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtiktoken\u001b[39;00m\n\u001b[0;32m----> 9\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mllama_index\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mllms\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m AzureOpenAI\n\u001b[1;32m     12\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mllama_index\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mtext_splitter\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m TokenTextSplitter\n\u001b[1;32m     13\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mllama_index\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mllms\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m AzureOpenAI , OpenAI\n",
      "\u001b[0;31mImportError\u001b[0m: cannot import name 'AzureOpenAI' from 'llama_index.llms' (unknown location)"
     ]
    }
   ],
   "source": [
    "#Download Dataset(https://www.kaggle.com/datasets/hinepo/harry-potter-books-in-pdf-1-7)\n",
    "\n",
    "# Import libraries\n",
    "\n",
    "import os \n",
    "from pathlib import Path\n",
    "import tiktoken\n",
    "\n",
    "from llama_index.llms import AzureOpenAI\n",
    "\n",
    "\n",
    "from llama_index.text_splitter import TokenTextSplitter\n",
    "from llama_index.llms import AzureOpenAI , OpenAI\n",
    "from llama_index import ServiceContext, PromptHelper\n",
    "\n",
    "from llama_index import VectorStoreIndex, SimpleDirectoryReader\n",
    "from llama_index import set_global_service_context\n",
    "\n",
    "from llama_index import Document\n",
    "\n",
    "\n",
    "import logging\n",
    "import sys\n",
    "\n",
    "from IPython.display import Markdown, display\n",
    "\n",
    "logging.basicConfig(\n",
    "    stream=sys.stdout, level=logging.INFO\n",
    ")  # logging.DEBUG for more verbose output\n",
    "logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 2: Load Documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "pypdf is required to read PDF files: `pip install pypdf`",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "File \u001b[0;32m~/miniconda3/envs/llamaindex_nebulagraph_phillies/lib/python3.11/site-packages/llama_index/readers/file/docs_reader.py:29\u001b[0m, in \u001b[0;36mPDFReader.load_data\u001b[0;34m(self, file, extra_info)\u001b[0m\n\u001b[1;32m     28\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 29\u001b[0m     \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpypdf\u001b[39;00m\n\u001b[1;32m     30\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m:\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'pypdf'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "File \u001b[0;32m~/miniconda3/envs/llamaindex_nebulagraph_phillies/lib/python3.11/site-packages/llama_index/readers/file/base.py:251\u001b[0m, in \u001b[0;36mSimpleDirectoryReader.load_data\u001b[0;34m(self, show_progress)\u001b[0m\n\u001b[1;32m    250\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 251\u001b[0m     docs \u001b[38;5;241m=\u001b[39m \u001b[43mreader\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43minput_file\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mextra_info\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmetadata\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    252\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    253\u001b[0m     \u001b[38;5;66;03m# ensure that ImportError is raised so user knows\u001b[39;00m\n\u001b[1;32m    254\u001b[0m     \u001b[38;5;66;03m# about missing dependencies\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/llamaindex_nebulagraph_phillies/lib/python3.11/site-packages/llama_index/readers/file/docs_reader.py:31\u001b[0m, in \u001b[0;36mPDFReader.load_data\u001b[0;34m(self, file, extra_info)\u001b[0m\n\u001b[1;32m     30\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m:\n\u001b[0;32m---> 31\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(\n\u001b[1;32m     32\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpypdf is required to read PDF files: `pip install pypdf`\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     33\u001b[0m     )\n\u001b[1;32m     34\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mopen\u001b[39m(file, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrb\u001b[39m\u001b[38;5;124m\"\u001b[39m) \u001b[38;5;28;01mas\u001b[39;00m fp:\n\u001b[1;32m     35\u001b[0m     \u001b[38;5;66;03m# Create a PDF object\u001b[39;00m\n",
      "\u001b[0;31mImportError\u001b[0m: pypdf is required to read PDF files: `pip install pypdf`",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[3], line 9\u001b[0m\n\u001b[1;32m      4\u001b[0m os\u001b[38;5;241m.\u001b[39menviron[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mOPENAI_API_VERSION\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m2023-06-01-preview\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m      6\u001b[0m \u001b[38;5;66;03m# Openapi 配置\u001b[39;00m\n\u001b[1;32m      7\u001b[0m \u001b[38;5;66;03m# os.environ['OPENAI_API_KEY'] = \"\"\u001b[39;00m\n\u001b[0;32m----> 9\u001b[0m documents \u001b[38;5;241m=\u001b[39m \u001b[43mSimpleDirectoryReader\u001b[49m\u001b[43m(\u001b[49m\u001b[43minput_dir\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mdata\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mload_data\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     11\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m docs is a \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mtype\u001b[39m(documents[\u001b[38;5;241m0\u001b[39m])\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m,  where each element is a \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mtype\u001b[39m(documents[\u001b[38;5;241m0\u001b[39m])\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m object\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m     13\u001b[0m \u001b[38;5;66;03m# Output\u001b[39;00m\n\u001b[1;32m     14\u001b[0m \u001b[38;5;66;03m# docs is a <class 'llama_index.schema.Document'>,  where each element is a <class 'llama_index.schema.Document'> object\u001b[39;00m\n",
      "File \u001b[0;32m~/miniconda3/envs/llamaindex_nebulagraph_phillies/lib/python3.11/site-packages/llama_index/readers/file/base.py:255\u001b[0m, in \u001b[0;36mSimpleDirectoryReader.load_data\u001b[0;34m(self, show_progress)\u001b[0m\n\u001b[1;32m    251\u001b[0m     docs \u001b[38;5;241m=\u001b[39m reader\u001b[38;5;241m.\u001b[39mload_data(input_file, extra_info\u001b[38;5;241m=\u001b[39mmetadata)\n\u001b[1;32m    252\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    253\u001b[0m     \u001b[38;5;66;03m# ensure that ImportError is raised so user knows\u001b[39;00m\n\u001b[1;32m    254\u001b[0m     \u001b[38;5;66;03m# about missing dependencies\u001b[39;00m\n\u001b[0;32m--> 255\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(\u001b[38;5;28mstr\u001b[39m(e))\n\u001b[1;32m    256\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    257\u001b[0m     \u001b[38;5;66;03m# otherwise, just skip the file and report the error\u001b[39;00m\n\u001b[1;32m    258\u001b[0m     \u001b[38;5;28mprint\u001b[39m(\n\u001b[1;32m    259\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFailed to load file \u001b[39m\u001b[38;5;132;01m{\u001b[39;00minput_file\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m with error: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m. Skipping...\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m    260\u001b[0m         flush\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m    261\u001b[0m     )\n",
      "\u001b[0;31mImportError\u001b[0m: pypdf is required to read PDF files: `pip install pypdf`"
     ]
    }
   ],
   "source": [
    "# AzureOpenAI 配置\n",
    "# os.environ['AZURE_OPENAI_API_KEY'] = \"xxx\"\n",
    "# os.environ['AZURE_OPENAI_ENDPOINT'] = \"https://xxx.openai.azure.com\"\n",
    "# os.environ['OPENAI_API_VERSION'] = \"xxx\"\n",
    "\n",
    "# Openapi 配置\n",
    "# os.environ['OPENAI_API_KEY'] = \"\"\n",
    "\n",
    "documents = SimpleDirectoryReader(input_dir='data').load_data()\n",
    "\n",
    "print(f\" docs is a {type(documents[0])},  where each element is a {type(documents[0])} object\")\n",
    "\n",
    "# Output\n",
    "# docs is a <class 'llama_index.schema.Document'>,  where each element is a <class 'llama_index.schema.Document'> object"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 3: Creating Text Chunks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1022\n"
     ]
    }
   ],
   "source": [
    "text_splitter = TokenTextSplitter(\n",
    "  separator=\" \",\n",
    "  chunk_size=1024,\n",
    "  chunk_overlap=20,\n",
    "  backup_separators=[\"\\n\"]\n",
    "\n",
    ")\n",
    "\n",
    "# optional\n",
    "nodes = text_splitter.get_nodes_from_documents([Document(text=\"long text\")], show_progress=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 4: Creating Knowledge Repositories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# AzureOpenAI 配置\n",
    "llm = AzureOpenAI(engine='gpt-35-turbo-16k',  temperature=0, max_tokens=256,use_azure_ad=False,timeout=60000)\n",
    "# Openapi 配置\n",
    "# llm = OpenAI(model='gpt-3.5-turbo', temperature=0, max_tokens=256)\n",
    "\n",
    "\n",
    "\n",
    "prompt_helper = PromptHelper(\n",
    "\n",
    "  context_window=4096,\n",
    "\n",
    "  num_output=256,\n",
    "\n",
    "  chunk_overlap_ratio=0.1,\n",
    "\n",
    "  chunk_size_limit=None\n",
    "\n",
    ")\n",
    "\n",
    "service_context = ServiceContext.from_defaults(\n",
    "\n",
    "  llm=llm,\n",
    "\n",
    "   embed_model=\"local:BAAI/bge-small-en-v1.5\",\n",
    "\n",
    "  # text_splitter=text_splitter,\n",
    "\n",
    "  prompt_helper=prompt_helper\n",
    "\n",
    ")\n",
    "\n",
    "set_global_service_context(service_context)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 5: Neo4j as the Vector Database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1009\n",
      "1009\n",
      "1009\n",
      "1009\n",
      "1009\n"
     ]
    }
   ],
   "source": [
    "index = VectorStoreIndex.from_documents(\n",
    "    documents,\n",
    "    service_context = service_context\n",
    "    )\n",
    "index.storage_context.persist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 6: Query Index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 请求会超时\n",
    "query_engine = index.as_query_engine(service_context=service_context)\n",
    "response = query_engine.query(\"Who did Harry Potter kill to be victorious\")\n",
    "print(response)\n",
    "\n",
    "# Output\n",
    "# Based on the context information provided, it is not stated that Harry Potter killed anyone to be victorious"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 7: Initiate the Neo4j vector wrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.vector_stores import Neo4jVectorStore\n",
    "\n",
    "username = \"neo4j\"\n",
    "password = \"pleaseletmein\"\n",
    "url = \"bolt://localhost:7687\"\n",
    "embed_dim = 1536\n",
    "\n",
    "neo4j_vector = Neo4jVectorStore(username, password, url, embed_dim)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 8 : Build the VectorStoreIndex with Graph Database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.storage.storage_context import StorageContext\n",
    "from IPython.display import Markdown, display\n",
    "\n",
    "# create storage context\n",
    "storage_context = StorageContext.from_defaults(vector_store=neo4j_vector)\n",
    "\n",
    "# create index\n",
    "index = VectorStoreIndex.from_documents(documents, storage_context=storage_context)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 9: Query the index:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "query_engine = index.as_query_engine()\n",
    "response = query_engine.query(\"Who is Harry Potter\")\n",
    "display(Markdown(f\"<b>{response}</b>\"))\n",
    "\n",
    "#Output \n",
    "# Harry Potter is a character in the book \"Harry Potter and the Half-Blood Prince\" by J.K. Rowling.\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
